package engine

import (
	"fmt"
	"cn.hughie/learngo/crawler1/fetcher"
)

type ConcurrentEngine struct {
	ItemServer chan interface{}
	Scheduler   Scheduler
	WorkerCount int
}

type Scheduler interface {
	Notifier
	Submit(Request)
	WorkChan() chan Request
	Run()
}

type Notifier interface {
	WorkerReady(chan Request)
}

//引擎入口
func (e *ConcurrentEngine) Go(request ...Request) {
	//回传 channel
	out := make(chan ParseResult)
	e.Scheduler.Run()
	//创建worker
	for i := 0; i < e.WorkerCount; i++ {
		e.createWoker(out, e.Scheduler.WorkChan(), e.Scheduler)
	}
	//加任务进去
	for _, r := range request {
		if urlContainer()(r.Url) {
			continue
		}
		e.Scheduler.Submit(r)
	}
	for {
		result := <-out
		//处理item
		go func() {
			for _,item := range result.Items{
				e.ItemServer <- item
			}
		}()
		for _, r := range result.Requests {
			if urlContainer()(r.Url) {
				continue
			}
			e.Scheduler.Submit(r)
		}
	}

}

//创建woker
func (e *ConcurrentEngine) createWoker(out chan ParseResult, in chan Request, ready Notifier) {
	go func() {
		for {
			ready.WorkerReady(in)
			result, err := work(<-in)
			if (err != nil) {
				continue
			}
			out <- result
		}

	}()
}

//closure 校验url
func urlContainer() func(string) bool {
	urls := make(map[string]bool)
	return func(url string) bool {
		if urls[url] {
			return true
		}
		urls[url] = true
		return false
	}
}

//fetch +
func work(request Request) (ParseResult, error) {
	contents, e := fetcher.Fetch(request.Url)
	if e != nil {
		return ParseResult{}, e
	}
	result := request.ParseFunc(contents)
	for _, item := range result.Items {
		fmt.Printf("Got item %v \n", item)
	}
	return result, nil
}
